#!/bin/bash

# K8s健康检查脚本 - 增强版
# 检查K8s集群和组件的健康状态

set -e

# 配置变量
LOG_FILE="/var/log/k8s-health-check.log"
CHECK_INTERVAL=30  # 检查间隔（秒）

# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# 日志函数
log() {
    local level=$1
    shift
    local message="$*"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    
    case $level in
        "INFO")
            echo -e "${BLUE}[INFO]${NC} $message" | tee -a $LOG_FILE
            ;;
        "SUCCESS")
            echo -e "${GREEN}[SUCCESS]${NC} $message" | tee -a $LOG_FILE
            ;;
        "WARNING")
            echo -e "${YELLOW}[WARNING]${NC} $message" | tee -a $LOG_FILE
            ;;
        "ERROR")
            echo -e "${RED}[ERROR]${NC} $message" | tee -a $LOG_FILE
            ;;
    esac
}

# 检查命令是否存在
check_command() {
    local cmd=$1
    if command -v $cmd &> /dev/null; then
        log "SUCCESS" "✓ $cmd 已安装: $(which $cmd)"
        local version=$($cmd --version 2>/dev/null || $cmd version 2>/dev/null || echo "版本信息获取失败")
        log "INFO" "  版本: $version"
        return 0
    else
        log "ERROR" "✗ $cmd 未安装"
        return 1
    fi
}

# 检查服务状态
check_service() {
    local service=$1
    if systemctl is-active --quiet $service; then
        log "SUCCESS" "✓ $service 服务正在运行"
        return 0
    else
        log "ERROR" "✗ $service 服务未运行"
        return 1
    fi
}

# 检查端口是否监听
check_port() {
    local port=$1
    local service=$2
    if netstat -tlnp | grep -q ":$port "; then
        log "SUCCESS" "✓ $service 端口 $port 正在监听"
        return 0
    else
        log "ERROR" "✗ $service 端口 $port 未监听"
        return 1
    fi
}

# 检查Docker镜像
check_images() {
    log "INFO" "检查Docker镜像..."
    local k8s_images=$(docker images | grep -E "(k8s\.gcr\.io|pause|etcd|coredns)" | wc -l)
    if [ $k8s_images -gt 0 ]; then
        log "SUCCESS" "✓ 找到 $k8s_images 个K8s相关镜像"
        docker images | grep -E "(k8s\.gcr\.io|pause|etcd|coredns)" | while read line; do
            log "INFO" "  $line"
        done
    else
        log "ERROR" "✗ 未找到K8s相关镜像"
        return 1
    fi
}

# 检查CNI插件
check_cni() {
    log "INFO" "检查CNI插件..."
    if [ -d "/opt/cni/bin" ] && [ "$(ls /opt/cni/bin | wc -l)" -gt 0 ]; then
        log "SUCCESS" "✓ CNI插件已安装"
        log "INFO" "CNI插件列表:"
        ls -la /opt/cni/bin/ | while read line; do
            log "INFO" "  $line"
        done
        return 0
    else
        log "ERROR" "✗ CNI插件未安装"
        return 1
    fi
}

# 检查K8s集群状态
check_cluster() {
    log "INFO" "检查K8s集群状态..."
    
    # 检查kubectl配置
    if [ ! -f "$HOME/.kube/config" ]; then
        log "WARNING" "未找到kubectl配置文件，可能不是master节点"
        return 0
    fi
    
    # 检查集群连接
    if kubectl cluster-info &> /dev/null; then
        log "SUCCESS" "✓ 集群连接正常"
        
        # 显示集群信息
        log "INFO" "集群信息:"
        kubectl cluster-info | while read line; do
            log "INFO" "  $line"
        done
        
        # 检查节点状态
        log "INFO" "节点状态:"
        kubectl get nodes -o wide | while read line; do
            log "INFO" "  $line"
        done
        
        # 检查Pod状态
        log "INFO" "系统Pod状态:"
        kubectl get pods -n kube-system -o wide | while read line; do
            log "INFO" "  $line"
        done
        
        # 检查是否有失败的Pod
        local failed_pods=$(kubectl get pods -n kube-system --field-selector=status.phase!=Running,status.phase!=Succeeded | grep -v NAME | wc -l)
        if [ $failed_pods -gt 0 ]; then
            log "WARNING" "发现 $failed_pods 个非正常状态的Pod"
            kubectl get pods -n kube-system --field-selector=status.phase!=Running,status.phase!=Succeeded | while read line; do
                log "WARNING" "  $line"
            done
        else
            log "SUCCESS" "✓ 所有系统Pod运行正常"
        fi
        
        return 0
    else
        log "ERROR" "✗ 无法连接到集群"
        return 1
    fi
}

# 检查网络连接
check_network() {
    log "INFO" "检查网络连接..."
    
    # 检查DNS解析
    if nslookup kubernetes.default.svc.cluster.local &> /dev/null; then
        log "SUCCESS" "✓ DNS解析正常"
    else
        log "WARNING" "DNS解析可能有问题"
    fi
    
    # 检查网络插件
    if kubectl get pods -n kube-system | grep -q "flannel\|calico\|weave"; then
        log "SUCCESS" "✓ 网络插件已安装"
    else
        log "WARNING" "未检测到网络插件，集群网络可能不可用"
    fi
}

# 检查资源使用情况
check_resources() {
    log "INFO" "检查资源使用情况..."
    
    # 检查内存使用
    local mem_usage=$(free | awk 'NR==2{printf "%.1f", $3*100/$2}')
    log "INFO" "内存使用率: ${mem_usage}%"
    if (( $(echo "$mem_usage > 90" | bc -l) )); then
        log "WARNING" "内存使用率过高: ${mem_usage}%"
    fi
    
    # 检查磁盘使用
    local disk_usage=$(df / | awk 'NR==2{print $5}' | sed 's/%//')
    log "INFO" "磁盘使用率: ${disk_usage}%"
    if [ $disk_usage -gt 90 ]; then
        log "WARNING" "磁盘使用率过高: ${disk_usage}%"
    fi
    
    # 检查CPU负载
    local load_avg=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | sed 's/,//')
    log "INFO" "系统负载: $load_avg"
    if (( $(echo "$load_avg > 5" | bc -l) )); then
        log "WARNING" "系统负载过高: $load_avg"
    fi
}

# 检查日志错误
check_logs() {
    log "INFO" "检查系统日志..."
    
    # 检查kubelet日志
    if journalctl -u kubelet --since "1 hour ago" | grep -i error | wc -l | grep -q "^0$"; then
        log "SUCCESS" "✓ kubelet日志无错误"
    else
        log "WARNING" "kubelet日志中发现错误"
        journalctl -u kubelet --since "1 hour ago" | grep -i error | head -5 | while read line; do
            log "WARNING" "  $line"
        done
    fi
    
    # 检查containerd日志
    if journalctl -u containerd --since "1 hour ago" | grep -i error | wc -l | grep -q "^0$"; then
        log "SUCCESS" "✓ containerd日志无错误"
    else
        log "WARNING" "containerd日志中发现错误"
        journalctl -u containerd --since "1 hour ago" | grep -i error | head -5 | while read line; do
            log "WARNING" "  $line"
        done
    fi
}

# 生成健康报告
generate_report() {
    local report_file="/tmp/k8s-health-report-$(date +%Y%m%d-%H%M%S).txt"
    
    log "INFO" "生成健康报告: $report_file"
    
    {
        echo "K8s集群健康检查报告"
        echo "生成时间: $(date)"
        echo "主机名: $(hostname)"
        echo "操作系统: $(cat /etc/os-release | grep PRETTY_NAME | cut -d'=' -f2 | tr -d '\"')"
        echo "=========================================="
        echo ""
        
        echo "系统资源:"
        echo "内存使用:"
        free -h
        echo ""
        echo "磁盘使用:"
        df -h
        echo ""
        echo "系统负载:"
        uptime
        echo ""
        
        echo "服务状态:"
        systemctl status kubelet containerd --no-pager
        echo ""
        
        if [ -f "$HOME/.kube/config" ]; then
            echo "集群状态:"
            kubectl get nodes
            echo ""
            kubectl get pods -n kube-system
            echo ""
        fi
        
    } > $report_file
    
    log "SUCCESS" "健康报告已生成: $report_file"
}

# 持续监控模式
monitor_mode() {
    log "INFO" "进入持续监控模式，每${CHECK_INTERVAL}秒检查一次..."
    log "INFO" "按Ctrl+C退出监控"
    
    while true; do
        echo ""
        log "INFO" "=== 健康检查 $(date) ==="
        
        # 基本检查
        check_command kubectl
        check_command kubeadm
        check_command kubelet
        check_service containerd
        check_service kubelet
        
        # 集群检查（如果是master节点）
        if [ -f "$HOME/.kube/config" ]; then
            check_cluster
        fi
        
        # 资源检查
        check_resources
        
        sleep $CHECK_INTERVAL
    done
}

# 显示使用说明
show_usage() {
    echo "K8s健康检查脚本"
    echo ""
    echo "用法:"
    echo "  $0                    # 执行一次完整检查"
    echo "  $0 --monitor          # 持续监控模式"
    echo "  $0 --report            # 生成健康报告"
    echo "  $0 --help             # 显示帮助信息"
    echo ""
    echo "参数:"
    echo "  --monitor            持续监控模式"
    echo "  --report             生成详细健康报告"
    echo "  --interval=SECONDS   监控间隔时间（默认30秒）"
}

# 主函数
main() {
    log "INFO" "开始K8s健康检查..."
    
    # 基本组件检查
    log "INFO" "检查基本组件..."
    check_command kubectl
    check_command kubeadm
    check_command kubelet
    check_command containerd
    check_command crictl
    check_command helm
    
    # 服务状态检查
    log "INFO" "检查服务状态..."
    check_service containerd
    check_service kubelet
    
    # 端口检查
    log "INFO" "检查端口监听..."
    check_port 6443 "Kubernetes API Server"
    check_port 10250 "Kubelet"
    
    # 镜像检查
    check_images
    
    # CNI检查
    check_cni
    
    # 集群检查
    check_cluster
    
    # 网络检查
    check_network
    
    # 资源检查
    check_resources
    
    # 日志检查
    check_logs
    
    log "SUCCESS" "健康检查完成！"
}

# 解析命令行参数
MONITOR_MODE=false
GENERATE_REPORT=false

while [[ $# -gt 0 ]]; do
    case $1 in
        --monitor)
            MONITOR_MODE=true
            shift
            ;;
        --report)
            GENERATE_REPORT=true
            shift
            ;;
        --interval=*)
            CHECK_INTERVAL="${1#*=}"
            shift
            ;;
        --help)
            show_usage
            exit 0
            ;;
        *)
            echo "未知参数: $1"
            show_usage
            exit 1
            ;;
    esac
done

# 执行相应操作
if [ "$GENERATE_REPORT" = true ]; then
    generate_report
elif [ "$MONITOR_MODE" = true ]; then
    monitor_mode
else
    main
fi
